#!/usr/bin/perl -w

use Data::Dumper;
use File::Copy;

require '/org/lists.debian.org/archive-spam-removals/etc/review.conf';
die ("$0: /org/lists.debian.org/archive-spam-removals/etc/review.conf incorrect: $!") unless $SpamReviewConf::num_nominations;

umask 002;

die ("$0: needs to be run as user listweb\n") if ($ENV{'LOGNAME'} ne 'listweb');

$spamlog='/org/lists.debian.org/spam-log/spam-report.log.0';
$reviewlog='/org/lists.debian.org/spam-log/spam-review.log.0';
$cachedir='/org/lists.debian.org/archive-spam-removals/review';
$skipdir='/org/lists.debian.org/archive-spam-removals/spam-removals/web-removals';

if (-e "$cachedir/cache") {
  open (CACHE, "$cachedir/cache") or die ("$0: can't open $cachedir/cache for reading: $!\n");
  while (<CACHE>) {
    chomp;
    ($post, $count, $msgid, $ham, $spam, $inappropriate) = split (/:/);
    $count{$post} = $count;
    $msgid{$post} = $msgid;
    @{$reviewed{$post}{'Ham'}} = split(',', $ham);
    @{$reviewed{$post}{'Spam'}} = split(',', $spam);;
    @{$reviewed{$post}{'Inappropriate'}} = split(',', $inappropriate);;
    $list = (split(m#\/#, $post))[0];
    $lists{$list} = 1;
    $listcount{$list} = 0;
  }
  close (CACHE);
}

if (-e $spamlog) {
  open (SPAMLOG, $spamlog) or die ("$0: can't open $spamlog for reading: $!\n");
  while (<SPAMLOG>) {
    if (m#time: (\d+); addr: (.+); list: ([a-z0-9-]+), date: (\d+/?\d*), msg: (msg\d+.html)#) {
      next if $nominated{"$3/$4/$5"}{$2};
      $nominated{"$3/$4/$5"}{$2} = 1;
      $count{"$3/$4/$5"}++;
      $msgid{"$3/$4/$5"} = lookup_msgid("$3/$4/$5") unless $msgid{"$3/$4/$5"};
      next unless $msgid{"$3/$4/$5"};
      next if $msgid{"$3/$4/$5"} =~ m# #; 
      $lists{$3} = 1;
      $listcount{$3} = 0;
    } else {
      warn("$0: can't parse line: $_");
    }
  }
  close (SPAMLOG);
}

if (-e $reviewlog) {
  open (REVIEWLOG, $reviewlog) or die ("$0: can't open $reviewlog for reading: $!\n");
  while (<REVIEWLOG>) {
    if (m#time: (\d+); addr: (\d+\.\d+\.\d+\.\d+); list: ([a-z0-9-]+), date: (\d+/\d*), msg: (msg\d+.html), rating: (Unsure|Spam|Ham|Inappropriate), reviewer: (\S+)#) {
      push (@{$reviewed{"$3/$4/$5"}{$6}}, $7) unless grep(m#^$7$#, @{$reviewed{"$3/$4/$5"}{$6}});
      $lists{$3} = 1;
      $listcount{$3} = 0;
    } elsif (m#time: (\d+); addr: ([\da-f:]+); list: ([a-z0-9-]+), date: (\d+/\d*), msg: (msg\d+.html), rating: (Unsure|Spam|Ham|Inappropriate), reviewer: (\S+)#) {
      push (@{$reviewed{"$3/$4/$5"}{$6}}, $7) unless grep(m#^$7$#, @{$reviewed{"$3/$4/$5"}{$6}});
      $lists{$3} = 1;
      $listcount{$3} = 0;
    } else {
      print "debug1: $_";
    }
  }
  close (REVIEWLOG);
}

open (CACHE, '>', "$cachedir/cache.new") or die ("$0: can't open $cachedir/cache.new for writing: $!\n");

foreach $list (sort keys %lists) {
  open ("$list", '>', "$cachedir/$list") or die ("$0: can't open $cachedir/$list for writing: $!\n");
}

foreach $post (sort {$count{$b} <=> $count{$a}} keys %count) {
  next unless $count{$post} > 0;
  next unless file_exists($post);
  $spam = $ham = $inappropriate = 0;
  $spam = scalar @{$reviewed{$post}{'Spam'}} if defined $reviewed{$post}{'Spam'};
  $ham = scalar @{$reviewed{$post}{'Ham'}} if defined $reviewed{$post}{'Ham'};
  $inappropriate = scalar @{$reviewed{$post}{'Inappropriate'}} if defined $reviewed{$post}{'Inappropriate'};
  print CACHE join(
		   ':',
		   $post,
		   $count{$post},
		   $msgid{$post} ? $msgid{$post} : '',
		   $ham  ? join (',', @{$reviewed{$post}{'Ham'}}) : '',
		   $spam ? join (',', @{$reviewed{$post}{'Spam'}}) : '',
		   $inappropriate ? join (',', @{$reviewed{$post}{'Inappropriate'}}) : '',
		   ) . "\n";
  next unless $msgid{$post};
  next if $msgid{$post} =~ m# #;
  if ($spam >= ($ham*$SpamReviewConf::minority_factor+$inappropriate*$SpamReviewConf::minority_factor+$SpamReviewConf::num_ratings)) {
    if ($post =~ m#^([-\w]+)/(\d+)/(\d*)/msg\d+.html$#) {
      $list = $1;
      $year = $2;
      $month = $3;
      push (@{$skip{"$list/$year/$list-$year$month.spam"}}, $msgid{$post});
    } else {
      print "debug2: $post\n";
    }
    next;
  }
  if ($ham >= ($spam*$SpamReviewConf::minority_factor+$inappropriate*$SpamReviewConf::minority_factor+$SpamReviewConf::num_ratings)) {
    if ($post =~ m#^([-\w]+)/(\d+)/(\d*)/msg\d+.html$#) {
      $list = $1;
      $year = $2;
      $month = $3;
      push (@{$skip{"$list/$year/$list-$year$month.ham"}}, $msgid{$post});
    } else {
      print "debug3: $post\n";
    }
    next;
  }

  if ($inappropriate >= ($ham*$SpamReviewConf::minority_factor+$spam*$SpamReviewConf::minority_factor+$SpamReviewConf::num_ratings)) {
    if ($post =~ m#^([-\w]+)/(\d+)/(\d*)/msg\d+.html$#) {
      $list = $1;
      $year = $2;
      $month = $3;
      push (@{$skip{"$list/$year/$list-$year$month.inappropriate"}}, $msgid{$post});
    } else {
      print "debug4: $post\n";
    }
    next;
  }
  next if $count{$post} < $SpamReviewConf::num_nominations;
  $list = (split('/', $post))[0];
  print $list join(
                   ':',
                   $post,
                   $count{$post},
                   $msgid{$post},
                   $ham  ? join (',', @{$reviewed{$post}{'Ham'}}) : '',
                   $spam ? join (',', @{$reviewed{$post}{'Spam'}}) : '',
                   $inappropriate ? join (',', @{$reviewed{$post}{'Inappropriate'}}) : '',
                   ) . "\n";
  $listcount{$list}++;
}

close (CACHE);

move("$cachedir/cache.new", "$cachedir/cache") or 
	die("$0: can't move $cachedir/cache.new to $cachedir/cache: $!\n");

foreach $list (keys %skip) {
  @file = split('/', $list);
  $file = $file[$#file];
  open (SKIPFILE, '>', "$skipdir/$file") or die ("$0: can't open $skipdir/$file for writing: $!\n");
  foreach $msgid (@{$skip{$list}}) {
    print SKIPFILE "Skip-Spam-Message-Id: <$msgid>\n";
  }
  close (SKIPFILE);
}

sub file_exists {
  my $post = shift(@_);
  if ($post =~ m#^([\w-]+)/(\d+)/(\d+)/(msg\d+.html)$#) {
    $file = '/org/lists.debian.org/www/' . $1 . '/' . $2 . '/' . $1 . '-' . $2 . $3 . '/' . $4;
  } elsif ($post =~ m#^([\w-]+)/(\d+)//(msg\d+.html)$#) {
    $file = '/org/lists.debian.org/www/' . $1 . '/' . $1 . '-' . $2 . '/' . $3;
  }
  if (-e $file) {
    return 1;
  } else {
    return 0;
  }
}

sub lookup_msgid {
  my $post = shift(@_);
  if ($post =~ m#^([\w-]+)/(\d+)/(\d+)/(msg\d+.html)$#) {
    $file = '/org/lists.debian.org/www/' . $1 . '/' . $2 . '/' . $1 . '-' . $2 . $3 . '/' . $4;
  } elsif ($post =~ m#^([\w-]+)/(\d+)/+(msg\d+.html)$#) {
    $file = '/org/lists.debian.org/www/' . $1 . '/' . $1 . '-' . $2 . '/' . $3;
  } else {
    warn("$0: does not match: $file\n");
  }
  if (-e $file) {
    open (FILE, "$file") or warn "$0: can't open $file for reading: $!\n";
    while (<FILE>) {
      s/&#45;/-/g;
      next unless m#^<!--X-Message-Id: (\S+) -->$#;
      close (FILE);
      my $msgid = $1;
      $msgid =~ s#:#&Colon;#g;
      return $msgid;
    }
  } else {
    return '';
  }
}
